Semantic Segmentation¶
Import Modules¶
In [1]:
# IMPORT MODULES
# Import Import Numpy, TensorFlow, Scipy, Keras
import sys
import time
import os
import numpy as np
import pandas as pd
from glob import glob
import cv2
import matplotlib.pyplot as plt
from skimage.segmentation import mark_boundaries
import pickle
import math
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from scipy.signal import find_peaks_cwt
import random
from sklearn.utils import shuffle
import csv
import tensorflow as tf
from tensorflow.contrib.layers import flatten
from tensorflow.examples.tutorials.mnist import input_data
import keras
from keras.datasets import cifar10
from keras.models import Sequential,model_from_json
from keras.layers import Dense,Dropout,Activation,Flatten
from keras.layers import Convolution2D,MaxPooling2D
from keras.layers import Flatten,Lambda,ELU
from keras.optimizers import SGD,Adam,RMSprop
from keras.layers.convolutional import Conv2D
from keras.layers.normalization import BatchNormalization
from keras.utils import np_utils
from keras.preprocessing.image import ImageDataGenerator
from keras.initializers import glorot_uniform
from sklearn.cross_validation import train_test_split
import scipy.misc
from matplotlib.pyplot import imshow
from IPython.display import SVG
import json
from keras.models import Sequential, model_from_json
from collections import namedtuple
from keras.models import Model, load_model
from keras.layers import Input, BatchNormalization, Dropout, Lambda
from keras.layers.convolutional import Conv2D, Conv2DTranspose
from keras.layers.pooling import MaxPooling2D
from keras.layers.merge import concatenate
from keras.layers import UpSampling2D
import time
from skimage import data, color
from skimage.transform import rescale, resize, downscale_local_mean
%matplotlib inline
print('Import Modules')
Load Data¶
In [2]:
# LOAD DATA
# Change Directory
os.chdir('..')
In [3]:
# LOAD DATA
# Load Training Data Directories
train_image_dir = "Data/Images/Train/"
train_label_dir = "Data/Ground-Truth/Train/"
train_dirs = ["jena/", "zurich/", "weimar/", "ulm/", "tubingen/", "stuttgart/",
"strasbourg/", "monchengladbach/", "krefeld/", "hanover/",
"hamburg/", "erfurt/", "dusseldorf/", "darmstadt/", "cologne/",
"bremen/", "bochum/", "aachen/"]
In [4]:
# LOAD DATA
# Define a Function to Convert Label Image to Binary Image
def create_binary_labels(img):
# Create Binary Image
binary_img = np.zeros((img.shape[0], img.shape[1]))
# Create For Loops for Check ground Pixels
for i in range(img.shape[0]):
for j in range(img.shape[1]):
if img[i,j][0] == 128 and img[i,j][1] == 64 and img[i,j][2] == 128:
binary_img[i,j] = 255
else:
binary_img[i,j] = 0
return binary_img
In [33]:
# LOAD DATA
# Define a Function to Perform Gamma Correction
def adjust_gamma(image, gamma=1.0):
invGamma = 1.0/gamma
table = np.array([((i/255.0) ** invGamma) * 255
for i in np.arange(0, 256)]).astype("uint8")
return cv2.LUT(image, table)
# Initialize Array
gamma_array = np.array([0.5, 0.65, 0.85, 1.0, 1.45, 1.75, 2.0])
In [6]:
# LOAD DATA
#Create Empty Array for Training Images
x_train = []
y_train = []
# Create a For Loop to Import Training Data
for dir_step, dir in enumerate(train_dirs):
# Load Current Directory
current_dir = train_image_dir + dir
# Load Filenames in Current Directory
filenames = os.listdir(current_dir)
# Create For Loop for Import Images from Current Directory
for step, filename in enumerate(filenames):
# Print Current Loading Status
if step % 10 == 0:
print("Training Data Directory: %d/%d, Step: %d/%d" % (dir_step, len(train_dirs)-1,step, \
len(filenames)-1))
# Check Image ID
image_id = filename.split("_left")[0]
# Read the Image
image_path = current_dir + filename
image = cv2.imread(image_path)
# Randomly Select Gamma
gamma = random.choice(gamma_array)
# Perform Gamma Correction
image = adjust_gamma(image, gamma = gamma)
# Resize the Image
image_small = cv2.resize(image, (144,144), interpolation=cv2.INTER_NEAREST)
# Flatten Image and Append
image_flat = image_small.flatten().tolist()
x_train += image_flat
# Read Label Image
label_image_path = train_label_dir + dir + image_id + "_gtFine_color.png"
label_image = cv2.imread(label_image_path)
# Resize the Label Image
label_image_small = cv2.resize(label_image, (144,144), interpolation=cv2.INTER_NEAREST)
# Convert to Binary Image
binary_label_image = create_binary_labels(label_image_small)
# Flatten Image and Append
label_image_flat = binary_label_image.flatten().tolist()
y_train += label_image_flat
In [7]:
# LOAD DATA
# Check Size of Data
image_feature_size=int(len(x_train))
print("Feature Size:", image_feature_size)
# Revert to the Original Image Shapes
x_train = np.array(x_train).reshape(2975, 144, 144, 3)
print("Feature Size:", x_train.shape)
# Check Size of Data
label_feature_size=int(len(y_train))
print("Feature Size:", label_feature_size)
# Revert to the Original Image Shapes
y_train = np.array(y_train).reshape(2975, 144, 144, 1)
print("Feature Size:", y_train.shape)
In [8]:
# LOAD DATA
# Plot Images
index = random.randint(0,2975)
fig, (axis1, axis2) = plt.subplots(1,2,figsize=(5,5))
axis1.imshow(x_train[index])
axis2.imshow(np.array(y_train[index]).reshape(144,144))
axis1.set_title('Original Image:',fontsize=10)
axis2.set_title('Label Image:',fontsize=10)
Out[8]:
In [9]:
# LOAD DATA
# Load Training Data Directories
valid_image_dir = "Data/Images/Val/"
valid_label_dir = "Data/Ground-Truth/Val/"
valid_dirs = ["frankfurt/", "lindau/", "munster/"]
In [37]:
# LOAD DATA
#Create Empty Array for Validation Images
x_valid = []
y_valid = []
# Create a For Loop to Import Validation Data
for dir_step, dir in enumerate(valid_dirs):
# Load Current Directory
current_dir = valid_image_dir + dir
# Load Filenames in Current Directory
filenames = os.listdir(current_dir)
# Create For Loop for Import Images from Current Directory
for step, filename in enumerate(filenames):
# Print Current Loading Status
if step % 10 == 0:
print("Validation Data Directory: %d/%d, Step: %d/%d" % (dir_step, len(valid_dirs)-1,step, \
len(filenames)-1))
# Check Image ID
image_id = filename.split("_left")[0]
# Read the Image
image_path = current_dir + filename
image = cv2.imread(image_path)
# Randomly Select Gamma
gamma = random.choice(gamma_array)
# Perform Gamma Correction
#image = adjust_gamma(image, gamma = gamma)
# Resize the Image
image_small = cv2.resize(image, (144,144), interpolation=cv2.INTER_NEAREST)
# Flatten Image and Append
image_flat = image_small.flatten().tolist()
x_valid += image_flat
# Read Label Image
label_image_path = valid_label_dir + dir + image_id + "_gtFine_color.png"
label_image = cv2.imread(label_image_path)
# Resize the Label Image
label_image_small = cv2.resize(label_image, (144,144), interpolation=cv2.INTER_NEAREST)
# Convert to Binary Image
binary_label_image = create_binary_labels(label_image_small)
# Flatten Image and Append
label_image_flat = binary_label_image.flatten().tolist()
y_valid += label_image_flat
In [39]:
# LOAD DATA
# Check Size of Data
image_feature_size=int(len(x_valid))
print("Feature Size:", image_feature_size)
# Revert to the Original Image Shapes
x_valid = np.array(x_valid).reshape(500, 144, 144, 3)
print("Feature Size:", x_valid.shape)
# Check Size of Data
label_feature_size=int(len(y_valid))
print("Feature Size:", label_feature_size)
# Revert to the Original Image Shapes
y_valid = np.array(y_valid).reshape(500, 144, 144, 1)
print("Feature Size:", y_valid.shape)
Create Model¶
In [12]:
# CREATE MODEL
# Define a Function for Convolutional Block
def convolutional_block(input_tensor, depth, kernel, strides=(1, 1), padding="SAME", batchnorm = False):
layer = tf.layers.conv2d(input_tensor, \
filters = depth, \
kernel_size = kernel, \
strides = strides, \
padding = padding, \
activation = tf.nn.relu)
layer = tf.layers.dropout(layer, rate = 0.20, training = True)
if batchnorm:
layer = tf.layers.batch_normalization(layer, training = False)
return layer
In [13]:
# CREATE MODEL
# Define a Function for Deconvolutional Block
def deconvolutional_block(input_tensor, filter_size, output_size, \
out_channels, in_channels, \
strides = [1, 1, 1, 1], name = False):
input_shape = tf.shape(input_tensor)
batch_size = input_shape[0]
out_shape = tf.stack([batch_size, output_size, output_size, out_channels])
filter_shape = [filter_size, filter_size, out_channels, in_channels]
weights = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.05))
if name:
layer = tf.nn.conv2d_transpose(input_tensor, weights, out_shape, strides, padding = 'SAME', name = 'y_pred')
else:
layer = tf.nn.conv2d_transpose(input_tensor, weights, out_shape, strides, padding = 'SAME')
return layer
In [14]:
# CREATE MODEL
# Define a Function for Architecture
def UNet(x, n_filters):
c1_a = convolutional_block(x, 16 * n_filters, 3)
c1 = convolutional_block(c1_a, 16 * n_filters, 3)
c1 = tf.nn.max_pool(value=c1,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
c2_a = convolutional_block(c1, 32 * n_filters, 3)
c2 = convolutional_block(c2_a, 32 * n_filters, 3)
c2 = tf.nn.max_pool(value=c2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
c3_a = convolutional_block(c2, 64 * n_filters, 3)
c3 = convolutional_block(c3_a, 64 * n_filters, 3)
c3 = tf.nn.max_pool(value=c3,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
c4_a = convolutional_block(c3, 128 * n_filters, 3)
c4 = convolutional_block(c4_a, 128 * n_filters, 3)
c4 = tf.nn.max_pool(value=c4,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
c5_a = convolutional_block(c4, 256 * n_filters, 3)
c5 = convolutional_block(c5_a, 256 * n_filters, 3)
d1 = deconvolutional_block(c5, 2, 18, 128 * n_filters, 256 * n_filters, strides=[1, 2, 2, 1])
cat1 = tf.concat([d1, c4_a], 3)
cat1 = convolutional_block(cat1, 128 * n_filters, 3)
cat1 = convolutional_block(cat1, 128 * n_filters, 3)
d2 = deconvolutional_block(cat1, 2, 36, 64 * n_filters, 128 * n_filters, strides=[1, 2, 2, 1])
cat2 = tf.concat([d2, c3_a], 3)
cat2 = convolutional_block(cat2, 64 * n_filters, 3)
cat2 = convolutional_block(cat2, 64 * n_filters, 3)
d3 = deconvolutional_block(cat2, 2, 72, 32 * n_filters, 64 * n_filters, strides=[1, 2, 2, 1])
cat3 = tf.concat([d3, c2_a], 3)
cat3 = convolutional_block(cat3, 32 * n_filters, 3)
cat3 = convolutional_block(cat3, 32 * n_filters, 3)
d4 = deconvolutional_block(cat3, 2, 144, 16 * n_filters, 32 * n_filters, strides=[1, 2, 2, 1])
cat4 = tf.concat([d4, c1_a], 3)
cat4 = convolutional_block(cat4, 16 * n_filters, 3)
cat4 = convolutional_block(cat4, 16 * n_filters, 3)
y_pred = deconvolutional_block(d4, 1, 144, 1, 16 * n_filters, name = True)
return y_pred
Set GPU Parameters¶
In [15]:
# SET GPU SETTINGS
# Configure
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allocator_type = 'BFC'
config.gpu_options.per_process_gpu_memory_fraction = 0.80
Train¶
In [16]:
# TRAIN
# Set Placeholder
x = tf.placeholder(tf.float32, shape=[None, 144, 144, 3], name = 'x')
y = tf.placeholder(tf.float32, [None, 144, 144, 1], name = 'y')
print("Setting Placeholder")
In [43]:
# TRAIN
# Set Training Pipeline
rate = 0.00001
batch_size = 8
epochs = 1000
y_pred = UNet(x,2)
loss_operation = tf.losses.mean_squared_error(y, y_pred)
optimizer = tf.train.AdamOptimizer(learning_rate = rate)
training_operation = optimizer.minimize(loss_operation)
print("Setting Training Pipeline")
In [44]:
# TRAIN
# Initialize Evaluation
correct_prediction = tf.losses.mean_squared_error(y, y_pred)
# Define Evaluation
def evaluate(x_data, y_data):
num_examples = len(x_data)
total_accuracy = 0
sess = tf.get_default_session()
for offset in range(0, num_examples, batch_size):
batch_x, batch_y = x_data[offset:offset + batch_size], y_data[offset:offset + batch_size]
accuracy = sess.run(correct_prediction, feed_dict={x:batch_x, y:batch_y})
total_accuracy += (accuracy * len(batch_x))
return total_accuracy/num_examples
In [45]:
# TRAIN
# Initialize Saver
saver = tf.train.Saver()
In [47]:
# TRAIN
# Initialize
training_loss_array = []
validation_loss_array = []
# Use Session to Test Model
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
num_examples = len(x_train)
print("Training...")
for i in range(epochs):
x_train, y_train = shuffle(x_train, y_train)
for offset in range(0, num_examples, batch_size):
end = offset + batch_size
batch_x, batch_y = x_train[offset:end], y_train[offset:end]
loss_value, _ = sess.run([loss_operation, training_operation], feed_dict={x:batch_x, y:batch_y})
training_loss_array.append(loss_value)
validation_loss = evaluate(x_valid, y_valid)
validation_loss_array.append(validation_loss)
saver.save(sess,'./Model-Tensorflow/unet')
print("Epoch {} ...".format(i+1))
print("Training Loss = {:.3f}".format(loss_value))
print("Validation Loss = {:.3f}".format(validation_loss))
print("Model Saved")
In [49]:
# TRAIN
# Plot Training Loss
plt.plot(training_loss_array)
plt.title('Training Loss')
Out[49]:
In [48]:
# TRAIN
# Plot Training Loss
plt.plot(validation_loss_array)
plt.title('Training Loss')
Out[48]:
Evaluate¶
In [22]:
# TEST
# Define a Function to Convert Label Image to Binary Image
def threshold_labels(img, thresh):
# Create Binary Image
binary_img = np.zeros((img.shape[0], img.shape[1]))
# Create For Loops for Check ground Pixels
for i in range(img.shape[0]):
for j in range(img.shape[1]):
if img[i,j] > thresh:
binary_img[i,j] = 255
else:
binary_img[i,j] = 0
return binary_img
In [36]:
# Evaluate the Model on Random Images
import time
# Use Session to Infer
with tf.Session() as sess:
saver.restore(sess,'Model-Tensorflow/unet')
for i in range(200,300):
# Predict
original_image = x_valid[i]
img = x_valid[i]
label_image = y_valid[i]
start_time = time.time()
original_image = np.reshape(original_image, [-1, 144, 144, 3])
test_data = {x:original_image}
test_mask = sess.run([y_pred],feed_dict=test_data)
test_mask = np.reshape(test_mask, (144,144))
end_time = time.time()
print('Computation Time:', end_time - start_time)
binary_image = threshold_labels(test_mask, 150)
# Resize Image
#img = cv2.resize(img, (1024, 2048))
#label_image = cv2.resize(label_image, (1024, 2048))
#binary_image = cv2.resize(binary_image, (1024, 2048))
# Plot
fig, (axis1, axis2, axis3) = plt.subplots(1, 3, figsize = (15,15))
axis1.imshow(img)
axis1.set_title('Input Image')
axis2.imshow(label_image[:,:,0])
axis2.set_title('Label')
axis3.imshow(binary_image)
axis3.set_title('Binary Prediction')
In [40]:
# Evaluate the Model on Random Images
import time
# Use Session to Infer
with tf.Session() as sess:
saver.restore(sess,'Model-Tensorflow/unet')
for i in range(200,300):
# Predict
original_image = x_valid[i]
img = x_valid[i]
label_image = y_valid[i]
start_time = time.time()
original_image = np.reshape(original_image, [-1, 144, 144, 3])
test_data = {x:original_image}
test_mask = sess.run([y_pred],feed_dict=test_data)
test_mask = np.reshape(test_mask, (144,144))
end_time = time.time()
print('Computation Time:', end_time - start_time)
binary_image = threshold_labels(test_mask, 150)
# Resize Image
#img = cv2.resize(img, (1024, 2048))
#label_image = cv2.resize(label_image, (1024, 2048))
#binary_image = cv2.resize(binary_image, (1024, 2048))
# Plot
fig, (axis1, axis2, axis3) = plt.subplots(1, 3, figsize = (15,15))
axis1.imshow(img)
axis1.set_title('Input Image')
axis2.imshow(label_image[:,:,0])
axis2.set_title('Label')
axis3.imshow(binary_image)
axis3.set_title('Binary Prediction')
In [ ]:
# TEST
# Import Modules
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import argparse
import sys
import tempfile
from tensorflow.examples.tutorials.mnist import input_data
import tensorflow as tf
from tensorflow.python.framework import graph_io
# Load Model
saver = tf.train.import_meta_graph('./Model-Tensorflow/unet.meta', clear_devices=True)
graph = tf.get_default_graph()
input_graph_def = graph.as_graph_def()
sess = tf.Session()
saver.restore(sess, "./Model-Tensorflow/unet")
print('Model Loaded!')
In [ ]:
# TEST
# Convert Model
output_node_names="y_pred"
output_graph_def = tf.graph_util.convert_variables_to_constants(sess, input_graph_def, output_node_names.split(","))
print('Model Converted!')
# Save Frozen Model
graph_io.write_graph(output_graph_def, './', './Model-Tensorflow/unet-benchmark.pb', as_text=False)
sess.close()
print('File Saved!')
In [ ]:
# TEST
# Load Model
frozen_graph="./Model-Tensorflow/unet-benchmark.pb"
with tf.gfile.GFile(frozen_graph, "rb") as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
with tf.Graph().as_default() as graph:
tf.import_graph_def(graph_def,
input_map=None,
return_elements=None,
name="")
y_pred = graph.get_tensor_by_name("y_pred:0")
# Start Session
sess= tf.Session(graph=graph)
# Predict in Loop
for i in range(400, 500):
# Load Images from Validation
original_image = x_valid[i]
img = x_valid[i]
label_image = y_valid[i]
# Resize the Image to the desired Size, Preprocessing will be done exactly as done during Training
original_image = np.reshape(original_image, [-1, 144, 144, 3])
# Initialize
images = []
images.append(original_image)
images = np.array(images, dtype=np.uint8)
images = images.astype('float32')
# NOTE: The input to the network is of shape [None image_size image_size num_channels].
# Reshape Image
x_batch = images.reshape(1, 144, 144, 3)
# Feed the Images to the Input Placeholders
x = graph.get_tensor_by_name("x:0")
# Create the 'feed_dict', Required to be fed to Calculate 'y_pred'
feed_dict_testing = {x: x_batch}
# Run Inference
start_time = time.time()
result=sess.run(y_pred, feed_dict=feed_dict_testing)
test_mask = np.array(result).reshape(144,144)
end_time = time.time()
print('Computation Time:', end_time - start_time)
binary_image = threshold_labels(test_mask, 100)
# Resize Image
label_image = resize(label_image, (200, 560), anti_aliasing=True)
binary_image = resize(binary_image, (200, 560), anti_aliasing=True)
# Plot
fig, (axis1, axis2, axis3) = plt.subplots(1, 3, figsize = (12,12))
axis1.imshow(image[:,:,0])
axis1.set_title('Input Image')
axis2.imshow(label_image[:,:,0])
axis2.set_title('Label Image')
axis3.imshow(binary_image)
axis3.set_title('Binary Prediction')